testomatio-reporter-cli 2.12.0 → 2.14.0-beta

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testomatio-reporter-cli",
-  "version": "2.12.0",
+  "version": "2.14.0-beta",
   "description": "Yarn Berry compatible standalone CLI for @testomatio/reporter",
   "type": "module",
   "bin": {

package/src/adapter/playwright.js

@@ -255,7 +255,8 @@ function appendStep(step, shift = 0) {
       newCategory = 'hook';
       break;
     case 'attach':
-      return null; // Skip steps with category 'attach'
+    case 'test.attach':
+      return null; // Attachments are reported as artifacts, not standalone steps
     default:
       newCategory = 'framework';
   }
@@ -286,11 +287,11 @@ function appendStep(step, shift = 0) {
     resultStep.log = truncate(String(step.log), 250);
   }
 
-  // Add artifacts from attachments
-  if (step.attachments && step.attachments.length > 0 && SCREENSHOTS_ON_STEPS) {
-    const screenshotAttachment = step.attachments.find(att =>
-      att.contentType === 'image/png' && att.name === 'screenshot'
-    );
+  // Playwright also associates automatic failure screenshots with the active
+  // hook. Only attachments created for an explicit test.step belong to a step;
+  // hook and fixture attachments remain test-level result artifacts.
+  if (step.category === 'test.step' && step.attachments?.length && SCREENSHOTS_ON_STEPS) {
+    const screenshotAttachment = step.attachments.find(isScreenshotArtifact);
     if (screenshotAttachment && screenshotAttachment.path) {
       const artifacts = { screenshot: screenshotAttachment.path };
       addArtifactsToStep(resultStep, artifacts);

package/src/bin/cli.js

@@ -62,7 +62,8 @@ program
     const apiKey = process.env['INPUT_TESTOMATIO-KEY'] || config.TESTOMATIO;
     const client = new TestomatClient({ apiKey });
 
-    const createRunParams = {};
+    // nothing is executed yet; the server flips it to running on the first reported test
+    const createRunParams = { status: 'scheduled' };
     if (opts.kind) createRunParams.kind = opts.kind;
 
     if (opts.filter) {

package/src/pipe/bitbucket.js

@@ -1,6 +1,13 @@
 import { APP_PREFIX, testomatLogoURL } from '../constants.js';
 import { ansiRegExp, isSameTest, truncate } from '../utils/utils.js';
-import { statusEmoji, fullName } from '../utils/pipe_utils.js';
+import {
+  statusEmoji,
+  fullName,
+  plannedTestsLabel,
+  markdownTable,
+  runSummary,
+  totalDuration,
+} from '../utils/pipe_utils.js';
 import { Gaxios } from 'gaxios';
 import pc from 'picocolors';
 import humanizeDuration from 'humanize-duration';
@@ -103,37 +110,32 @@ export class BitbucketPipe {
     }
 
     // Create a comment on Bitbucket
-    const passedCount = this.tests.filter(t => t.status === 'passed').length;
-    const failedCount = this.tests.filter(t => t.status === 'failed').length;
-    const skippedCount = this.tests.filter(t => t.status === 'skipped').length;
-
-    // Constructing the table
-    let summary = `${this.hiddenCommentData}
-    
-  | ![Testomat.io Report](${testomatLogoURL}) | ${statusEmoji(
-    runParams.status,
-  )} ${runParams.status.toUpperCase()} ${statusEmoji(runParams.status)} |
-  | --- | --- |
-  | **Tests** | ✔️ **${this.tests.length}** tests run |
-  | **Summary** | ${statusEmoji('failed')} **${failedCount}** failed; ${statusEmoji(
-    'passed',
-  )} **${passedCount}** passed; **${statusEmoji('skipped')}** ${skippedCount} skipped |
-  | **Duration** | 🕐 **${humanizeDuration(
-    parseInt(
-      this.tests.reduce((a, t) => a + (t.run_time || 0), 0),
-      10,
-    ),
-    {
-      maxDecimalPoints: 0,
-    },
-  )}** |
-  `;
+    // a scheduled run has no results yet: no counters, no duration
+    const isPendingRun = runParams.status === 'pending';
+
+    /** @type {Object<string, string>} */
+    const rows = {};
+
+    if (isPendingRun) {
+      if (this.tests.length) rows.Tests = `⚪ ${plannedTestsLabel(this.tests, this.store.runTestsCount)}`;
+    } else {
+      rows.Tests = `✔️ **${this.tests.length}** tests run`;
+      rows.Summary = runSummary(this.tests);
+      rows.Duration = `🕐 **${totalDuration(this.tests)}**`;
+    }
 
     if (this.ENV.BITBUCKET_BRANCH && this.ENV.BITBUCKET_COMMIT) {
-      // eslint-disable-next-line max-len
-      summary += `| **Job** | 👷 [#${this.ENV.BITBUCKET_BUILD_NUMBER}](https://bitbucket.org/${this.ENV.BITBUCKET_REPO_FULL_NAME}/pipelines/results/${this.ENV.BITBUCKET_BUILD_NUMBER}") by commit: **${this.ENV.BITBUCKET_COMMIT}** |`;
+      const buildNumber = this.ENV.BITBUCKET_BUILD_NUMBER;
+      const buildUrl = `https://bitbucket.org/${this.ENV.BITBUCKET_REPO_FULL_NAME}/pipelines/results/${buildNumber}`;
+      rows.Job = `👷 [#${buildNumber}](${buildUrl}) by commit: **${this.ENV.BITBUCKET_COMMIT}**`;
     }
 
+    const header = [
+      `![Testomat.io Report](${testomatLogoURL})`,
+      `${statusEmoji(runParams.status)} ${runParams.status.toUpperCase()} ${statusEmoji(runParams.status)}`,
+    ];
+    const summary = `${this.hiddenCommentData}\n\n${markdownTable(header, rows, { boldLabels: true })}`;
+
     const failures = this.tests
       .filter(t => t.status === 'failed')
       .slice(0, 20)
@@ -182,7 +184,7 @@ export class BitbucketPipe {
       }
     }
 
-    if (this.tests.length > 0) {
+    if (this.tests.length > 0 && !isPendingRun) {
       body += `\n\n**🐢 Slowest Tests**\n\n`;
       body += this.tests
         .sort((a, b) => b.run_time - a.run_time)

package/src/pipe/github.js

@@ -5,7 +5,14 @@ import humanizeDuration from 'humanize-duration';
 import merge from 'lodash.merge';
 import { testomatLogoURL } from '../constants.js';
 import { ansiRegExp, isSameTest, truncate } from '../utils/utils.js';
-import { statusEmoji, fullName } from '../utils/pipe_utils.js';
+import {
+  statusEmoji,
+  fullName,
+  plannedTestsLabel,
+  markdownTable,
+  runSummary,
+  totalDuration,
+} from '../utils/pipe_utils.js';
 import { log } from '../utils/log.js';
 
 const debug = createDebugMessages('@testomatio/reporter:pipe:github');
@@ -75,42 +82,37 @@ class GitHubPipe {
     if (!(owner || repo)) return;
 
     // ... create a comment on GitHub
-    const passedCount = this.tests.filter(t => t.status === 'passed').length;
-    const failedCount = this.tests.filter(t => t.status === 'failed').length;
-    const skippedCount = this.tests.filter(t => t.status === 'skipped').length;
-
-    let summary = `${this.hiddenCommentData}
-
-| [![Testomat.io Report](${testomatLogoURL})](https://testomat.io)  | ${statusEmoji(
-      runParams.status,
-    )} ${`${process.env.GITHUB_JOB} ${runParams.status}`.toUpperCase()} |
-| --- | --- |        
-| Tests | ✔️  **${this.tests.length}** tests run  |
-| Summary | ${failedCount ? `${statusEmoji('failed')} **${failedCount}** failed; ` : ''} ${statusEmoji(
-      'passed',
-    )} **${passedCount}** passed; **${statusEmoji('skipped')}** ${skippedCount} skipped |
-| Duration | 🕐  **${humanizeDuration(
-      parseInt(
-        this.tests.reduce((a, t) => a + (t.run_time || 0), 0),
-        10,
-      ),
-      {
-        maxDecimalPoints: 0,
-      },
-    )}** |`;
+    // a scheduled run has no results yet: no counters, no duration
+    const isPendingRun = runParams.status === 'pending';
+
+    /** @type {Object<string, string>} */
+    const rows = {};
+
+    if (isPendingRun) {
+      if (this.tests.length) rows.Tests = `⚪ ${plannedTestsLabel(this.tests, this.store.runTestsCount)}`;
+    } else {
+      rows.Tests = `✔️ **${this.tests.length}** tests run`;
+      rows.Summary = runSummary(this.tests);
+      rows.Duration = `🕐 **${totalDuration(this.tests)}**`;
+    }
 
     if (this.store.runUrl) {
-      summary += `\n| Testomat.io Report | 📊 [Run #${this.store.runId}](${this.store.runUrl})  | `;
+      rows['Testomat.io Report'] = `📊 [Run #${this.store.runId}](${this.store.runUrl})`;
     }
     if (process.env.GITHUB_WORKFLOW) {
-      summary += `\n| Job | 🗂️  [${this.jobKey}](${process.env.GITHUB_SERVER_URL || 'https://github.com'}/${
-        this.repo
-      }/actions/runs/${process.env.GITHUB_RUN_ID}) | `;
+      const server = process.env.GITHUB_SERVER_URL || 'https://github.com';
+      rows.Job = `🗂️ [${this.jobKey}](${server}/${this.repo}/actions/runs/${process.env.GITHUB_RUN_ID})`;
     }
     if (process.env.RUNNER_OS) {
-      summary += `\n| Operating System | 🖥️ \`${process.env.RUNNER_OS}\` ${process.env.RUNNER_ARCH || ''} | `;
+      rows['Operating System'] = `🖥️ \`${process.env.RUNNER_OS}\` ${process.env.RUNNER_ARCH || ''}`;
     }
 
+    const header = [
+      `[![Testomat.io Report](${testomatLogoURL})](https://testomat.io)`,
+      `${statusEmoji(runParams.status)} ${`${process.env.GITHUB_JOB} ${runParams.status}`.toUpperCase()}`,
+    ];
+    const summary = `${this.hiddenCommentData}\n\n${markdownTable(header, rows)}`;
+
     const failures = this.tests
       .filter(t => t.status === 'failed')
       .slice(0, 20)
@@ -170,7 +172,7 @@ class GitHubPipe {
       body += '\n\n</details>';
     }
 
-    if (this.tests.length > 0) {
+    if (this.tests.length > 0 && !isPendingRun) {
       body += '\n<details>\n<summary><h3>🐢 Slowest Tests</h3></summary>\n\n';
       body += this.tests
         .sort((a, b) => b?.run_time - a?.run_time)

package/src/pipe/gitlab.js

@@ -6,7 +6,14 @@ import merge from 'lodash.merge';
 import path from 'path';
 import { APP_PREFIX, testomatLogoURL } from '../constants.js';
 import { ansiRegExp, isSameTest, truncate } from '../utils/utils.js';
-import { statusEmoji, fullName } from '../utils/pipe_utils.js';
+import {
+  statusEmoji,
+  fullName,
+  plannedTestsLabel,
+  markdownTable,
+  runSummary,
+  totalDuration,
+} from '../utils/pipe_utils.js';
 import { log } from '../utils/log.js';
 
 const debug = createDebugMessages('@testomatio/reporter:pipe:gitlab');
@@ -81,37 +88,31 @@ class GitLabPipe {
     if (runParams.tests) runParams.tests.forEach(t => this.addTest(t));
 
     // ... create a comment on GitLab
-    const passedCount = this.tests.filter(t => t.status === 'passed').length;
-    const failedCount = this.tests.filter(t => t.status === 'failed').length;
-    const skippedCount = this.tests.filter(t => t.status === 'skipped').length;
-
-    // constructing the table
-    let summary = `${this.hiddenCommentData}
-    
-  | [![Testomat.io Report](${testomatLogoURL})](https://testomat.io)  | ${statusEmoji(
-    runParams.status,
-  )} ${runParams.status.toUpperCase()} ${statusEmoji(runParams.status)} |
-  | --- | --- |
-  | Tests | ✔️  **${this.tests.length}** tests run  |
-  | Summary | ${statusEmoji('failed')} **${failedCount}** failed; ${statusEmoji(
-    'passed',
-  )} **${passedCount}** passed; **${statusEmoji('skipped')}** ${skippedCount} skipped |
-  | Duration | 🕐  **${humanizeDuration(
-    parseInt(
-      this.tests.reduce((a, t) => a + (t.run_time || 0), 0),
-      10,
-    ),
-    {
-      maxDecimalPoints: 0,
-    },
-  )}** |
-  `;
+    // a scheduled run has no results yet: no counters, no duration
+    const isPendingRun = runParams.status === 'pending';
+
+    /** @type {Object<string, string>} */
+    const rows = {};
+
+    if (isPendingRun) {
+      if (this.tests.length) rows.Tests = `⚪ ${plannedTestsLabel(this.tests, this.store.runTestsCount)}`;
+    } else {
+      rows.Tests = `✔️ **${this.tests.length}** tests run`;
+      rows.Summary = runSummary(this.tests);
+      rows.Duration = `🕐 **${totalDuration(this.tests)}**`;
+    }
 
     if (this.ENV.CI_JOB_NAME && this.ENV.CI_JOB_ID) {
       // eslint-disable-next-line max-len
-      summary += `| Job | 👷 [${this.ENV.CI_JOB_ID}](${this.ENV.CI_JOB_URL})<br>Name: **${this.ENV.CI_JOB_NAME}**<br>Stage: **${this.ENV.CI_JOB_STAGE}** | `;
+      rows.Job = `👷 [${this.ENV.CI_JOB_ID}](${this.ENV.CI_JOB_URL})<br>Name: **${this.ENV.CI_JOB_NAME}**<br>Stage: **${this.ENV.CI_JOB_STAGE}**`;
     }
 
+    const header = [
+      `[![Testomat.io Report](${testomatLogoURL})](https://testomat.io)`,
+      `${statusEmoji(runParams.status)} ${runParams.status.toUpperCase()} ${statusEmoji(runParams.status)}`,
+    ];
+    const summary = `${this.hiddenCommentData}\n\n${markdownTable(header, rows)}`;
+
     const failures = this.tests
       .filter(t => t.status === 'failed')
       .slice(0, 20)
@@ -158,7 +159,7 @@ class GitLabPipe {
       body += '\n\n</details>';
     }
 
-    if (this.tests.length > 0) {
+    if (this.tests.length > 0 && !isPendingRun) {
       body += '\n<details>\n<summary><h3>🐢 Slowest Tests</h3></summary>\n\n';
       body += this.tests
         .sort((a, b) => b?.run_time - a?.run_time)

package/src/pipe/testomatio.js

@@ -323,6 +323,7 @@ class TestomatioPipe {
         shared_run: this.sharedRun,
         shared_run_timeout: this.sharedRunTimeout,
         kind: params.kind,
+        status: params.status,
         configuration,
         description,
         ci,
@@ -373,6 +374,8 @@ class TestomatioPipe {
       this.store.runUrl = this.runUrl;
       this.store.runPublicUrl = this.runPublicUrl;
       this.store.runId = this.runId;
+      // only the server knows how many tests a configuration expands to; automated runs report 0
+      if (resp.data.tests_count > 0) this.store.runTestsCount = resp.data.tests_count;
       log.info('📊 Report created. Report ID:', this.runId);
       process.env.runId = this.runId;
       debug('Run created', this.runId);

package/src/utils/pipe_utils.js

@@ -1,3 +1,4 @@
+import humanizeDuration from 'humanize-duration';
 import { log } from './log.js';
 
 /**
@@ -229,6 +230,80 @@ function formatFilterListIds(ids, format) {
   }
 }
 
+/**
+ * Summarize a finished run, e.g. `🔴 **1** failed; 🟢 **8** passed; 🟡 **1** skipped`.
+ * The failed part is omitted when nothing failed.
+ *
+ * @param {Array<{status?: string}>} tests
+ * @returns {string}
+ */
+function runSummary(tests) {
+  const countOf = status => tests.filter(t => t.status === status).length;
+  const failedCount = countOf('failed');
+
+  const parts = [];
+  if (failedCount) parts.push(`${statusEmoji('failed')} **${failedCount}** failed`);
+  parts.push(`${statusEmoji('passed')} **${countOf('passed')}** passed`);
+  parts.push(`${statusEmoji('skipped')} **${countOf('skipped')}** skipped`);
+
+  return parts.join('; ');
+}
+
+/**
+ * Total run time of the given tests, humanized — e.g. `2 seconds`.
+ *
+ * @param {Array<{run_time?: number}>} tests
+ * @returns {string}
+ */
+function totalDuration(tests) {
+  const milliseconds = tests.reduce((total, t) => total + (t.run_time || 0), 0);
+  return humanizeDuration(Math.trunc(milliseconds), { maxDecimalPoints: 0 });
+}
+
+/**
+ * Render a two-column markdown table. Rows with an empty value are skipped, so optional rows
+ * need no surrounding `if`.
+ *
+ * @param {string[]} header - The two header cells.
+ * @param {Object<string, string>} rows - Label to value, rendered in insertion order.
+ * @param {{ boldLabels?: boolean }} [opts] - Set `boldLabels` to wrap every label in `**`.
+ * @returns {string} The table, with no trailing newline.
+ */
+function markdownTable(header, rows, opts = {}) {
+  const lines = [`| ${header[0]} | ${header[1]} |`, '| --- | --- |'];
+
+  for (const [label, value] of Object.entries(rows)) {
+    if (!value) continue;
+
+    if (opts.boldLabels) {
+      lines.push(`| **${label}** | ${value} |`);
+    } else {
+      lines.push(`| ${label} | ${value} |`);
+    }
+  }
+
+  return lines.join('\n');
+}
+
+/**
+ * Describe the scope of a run that was prepared but not executed yet. Prefers the server's count;
+ * without it, falls back to the scoped ids, which mix tests (`T…`) and suites (`S…`).
+ *
+ * @param {Array<{test_id?: string}>} tests - Prepared tests the run was scoped to.
+ * @param {number} [testsCount] - Real number of tests, as reported by Testomat.io.
+ * @returns {string} Markdown label, e.g. `**159** tests planned` or `**6** suites planned`.
+ */
+function plannedTestsLabel(tests, testsCount) {
+  if (testsCount > 0) return `**${testsCount}** tests planned`;
+
+  const suitesCount = tests.filter(t => `${t.test_id || ''}`.startsWith('S')).length;
+  const knownTestsCount = tests.length - suitesCount;
+
+  if (!suitesCount) return `**${knownTestsCount}** tests planned`;
+  if (!knownTestsCount) return `**${suitesCount}** suites planned`;
+  return `**${knownTestsCount}** tests and **${suitesCount}** suites planned`;
+}
+
 export {
   updateFilterType,
   parseFilterParams,
@@ -236,6 +311,10 @@ export {
   setS3Credentials,
   statusEmoji,
   fullName,
+  markdownTable,
+  runSummary,
+  totalDuration,
+  plannedTestsLabel,
   parsePipeOptions,
   formatFilterListIds,
   getObjectSize,

package/types/types.d.ts

@@ -346,6 +346,9 @@ export interface CreateRunParams {
   /** Run configuration merged into the server-side run configuration. */
   configuration?: Record<string, any>;
 
+  /** Initial run status. `scheduled` marks a prepared run; the server flips it to running. */
+  status?: 'scheduled';
+
   /** Override batch upload mode. */
   batchMode?: BatchMode;
 }